package com.easy.stock.spider.html;

import java.io.IOException;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

import org.apache.commons.lang3.StringEscapeUtils;
import org.jboss.netty.handler.codec.http.HttpResponse;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;

import com.easy.stock.dao.StockFeedItemDao;
import com.easy.stock.dao.StockInfoDao;
import com.easy.stock.domain.StockFeedItem;
import com.easy.stock.domain.StockInfo;
import com.easy.stock.spider.HttpResponseHandler;
import com.easy.stock.spider.SpiderTask;
import com.easy.stock.spider.SpiderTask.STATUS;
import com.easy.stock.util.Constants;

public class StockTwitsHtmlSpider extends BaseHtmlSpider {

	@Autowired
	private StockInfoDao stockInfoDao;
	
	@Autowired
	private StockFeedItemDao stockFeedItemDao;
	
	private String baseUrl;
	
	public void setBaseUrl(String baseUrl) {
		this.baseUrl = baseUrl;
	}

	
	@Override
	protected List<SpiderTask> prepareSeeds() {
		
		List<SpiderTask> tasks = new ArrayList<SpiderTask>();
		List<StockInfo> stocks = null;
		int limit=1000;
		int offset=0;
		while(!(stocks=stockInfoDao.getStockInfoList(offset, limit)).isEmpty()) {
			for(StockInfo stock : stocks) {
				SpiderTask task = new SpiderTask(baseUrl+stock.getId()+"?q="+stock.getId(), new DetailHandler(this, stock.getId()), STATUS.READY);
				tasks.add(task);
			}
			
			offset+=limit;
		}
		return tasks;
		
////		// for test
//		List<SpiderTask> tasks = new ArrayList<SpiderTask>();
//		SpiderTask task = new SpiderTask(baseUrl+"CMGE"+"?q=CMGE", new DetailHandler(this, "CMGE"), STATUS.READY);
//		tasks.add(task);
//		return tasks;
	}
	

	
	public static class DetailHandler extends HttpResponseHandler {

		private String stockId;
		private SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy h:mm:ss", Locale.US);


		public DetailHandler(BaseHtmlSpider spider, String stockId) {
			super.spider = spider;
			this.stockId = stockId;
		}
		
		@Override
		public List<SpiderTask> doOnCompleted(HttpResponse response)
				throws IOException {
			
			StockTwitsHtmlSpider mySpider = (StockTwitsHtmlSpider)spider;
			String content = response.getContent().toString(Charset.forName("UTF-8"));
			Document doc = Jsoup.parse(content);
			Elements elems = doc.select("ol.stream-list li");
			int elemNum = elems.size();
			int feedNum = 0;
			for(int i=0; i<elemNum; i++) {
				Element elem = elems.get(i);
				
				String data = elem.attr("data-src");
				if(data==null || data.length()==0)
					continue;
				
				data = StringEscapeUtils.unescapeHtml4(data);
				JSONObject dataObj = new JSONObject(data);
				
				// 跳过评论部抓取
				if(!dataObj.isNull("in_reply_to_message_id"))
					continue;
				
				String msgId=Constants.STOCKTWIS+Long.toString(dataObj.getLong("id"));
				// 已经存在的不抓取
				if(mySpider.stockFeedItemDao.getFeedItem(msgId)!=null)
					continue;
				
				StockFeedItem feed = new StockFeedItem();
				feed.setId(msgId);
				feed.setSrc(Constants.STOCKTWIS);
				String dateStr = dataObj.getString("created_at").split("-")[0].trim();
				try {
					feed.setTime(sdf.parse(dateStr));
				} catch (ParseException e) {
					throw new RuntimeException("fail to parse stocktalk date, str="+dateStr);
				}
				feed.setContent(dataObj.getString("body"));
				feed.setStock_id(stockId);
				feed.setAuthor_name(dataObj.getJSONObject("user").getString("name"));
				if(!dataObj.isNull("sentiment")) {
					JSONObject sentiment = dataObj.getJSONObject("sentiment");
					feed.setAttitude(sentiment.getString("name"));
				}
				
				mySpider.stockFeedItemDao.addFeedItem(feed);
				feedNum++;
			}
			
			LOG.info("finish crawl stock twist page, stockId="+stockId+", feeds="+feedNum);
			return null;
		}
		
		
		@Override
		public void doOnThrowable(Throwable t) {
			// do nothing
		}
		
	}
}
